##================================================## ## ## ## Computational Social Science ## ## Doctoral Seminar (16:194:630) ## ## Spring 2026, Rutgers University ## ## Katya Ognyanova, katya@ognyanova.net ## ## ## ##================================================## # ================ Getting started: LLMs in R ================ # Packages we would need today: # (remove # and run this if they are not already installed) # install.packages("ellmer") # LLM API calls # install.packages("guardianapi") # Guardian newspaper API # install.packages("tidyverse") # Data processing # install.packages("caret") # Accuracy metrics # Run these lines in your console (replacing with real key): # Sys.setenv(OPENAI_API_KEY = "your_openai_key") # OpenAI # Sys.setenv(GOOGLE_API_KEY = "your_gemini_key") # Gemini # Check if your keys are available: nzchar(Sys.getenv("OPENAI_API_KEY")) # OpenAI nzchar(Sys.getenv("GOOGLE_API_KEY")) # Gemini library(ellmer) library(tidyverse) library(guardianapi) library(caret) # ================ Model Parameters ================ # Let's start by examining model parameters we can use in our work. # The params() function lets us control how the model generates text. # TEMPERATURE # Temperature controls randomness in the model's output. # Range: 0 to 2. Default is usually around 1. # Low temperature = more predictable. High = more creative (and variable). # Let's see this in action. We'll ask the same question 3 times # at the default temperature and compare the answers: question <- "In one sentence: What is the funniest ice cream flavor?" my_llm <- chat_openai(model = "gpt-4o-mini") my_llm$chat(question) my_llm <- chat_openai(model = "gpt-4o-mini") my_llm$chat(question) my_llm <- chat_openai(model = "gpt-4o-mini") my_llm$chat(question) # The model samples from a probability distribution, so the same prompt # can produce different results if we run it multiple times. # Now let's set the temperature parameter at 0 and run again: my_llm <- chat_openai(model = "gpt-4o-mini", params = params(temperature = 0)) my_llm$chat(question) my_llm <- chat_openai(model = "gpt-4o-mini", params = params(temperature = 0)) my_llm$chat(question) my_llm <- chat_openai(model = "gpt-4o-mini", params = params(temperature = 0)) my_llm$chat(question) # These should be much more consistent. Temperature 0 always picks the most # probable next token, so the output is going to be very similar across runs. # For research annotation, we often want to use temperature = 0 for consistency. # Note that even with temperature = 0, results are not 100% guaranteed # to be identical, but they're much more stable. # Let's try for creative answers: my_llm <- chat_openai(model = "gpt-4o-mini", params = params(temperature = 1.2)) my_llm$chat(question) my_llm <- chat_openai(model = "gpt-4o-mini", params = params(temperature = 1.5)) my_llm$chat(question) my_llm <- chat_openai(model = "gpt-4o-mini", params = params(temperature = 1.7)) my_llm$chat(question) # TOP P # Controls cumulative probability for token selection (so varies from 0 to 1) # Lower = more focused on high-probability tokens (similar effect to lower temperature) # Typically you would set either temperature OR top_p, not both. my_llm <- chat_openai(model = "gpt-4o-mini", params = params(top_p = 0.1)) my_llm$chat(question) my_llm <- chat_openai(model = "gpt-4o-mini", params = params(top_p = 0.9)) my_llm$chat(question) #TOP K # Similar, but focused on the number of highest-probability tokens to keep. # Select a positive integer between 1 and 20. my_llm <- chat_openai(model = "gpt-4o-mini", params = params(top_k = 20)) my_llm$chat(question) # FREQUENCY PENALTY # Penalizes tokens that have already appeared, reducing repetition. # Range: -2 to 2. Positive = less repetition. my_llm <- chat_openai(model = "gpt-4o-mini", params = params(frequency_penalty = 1.5)) my_llm$chat(question) # ================ Roles and Bias ================ # The system prompt sets the model's role for the conversation. # We can use it for task framing: "You are a helpful research assistant." # But roles can do much more; plus they can reveal some real challenges with # what LLMs have learned from their training data. # ------~~ Roles for task framing -------- # Roles shape the model's register, expertise level, and priorities. # Compare these two approaches to explaining the same concept: my_llm <- chat_openai(model = "gpt-4o-mini", system_prompt = "You are a computer science professor speaking at a seminar.") my_llm$chat("What are LLMs? Answer in 2 sentences.") my_llm <- chat_openai(model = "gpt-4o-mini", system_prompt = "You are a kindergarten teacher explaining ideas to kids.") my_llm$chat("What are LLMs? Answer in 2 sentences.") # ------~~ Demographic personas -------- # Another useful exercise for research is asking the LLM to respond # as a person with different specific demographic characteristics. # The model does this based on patterns in its training data # reflecting how people in those groups are represented in text. top_issue <- "What is the most important issue facing America today that you consider when voting in elections? Answer in 1 sentence." my_llm <- chat_openai(model = "gpt-4o-mini", system_prompt = "You are a 22-year-old Black woman, politically liberal, living in New York City, studying to be a nurse.") my_llm$chat(top_issue) my_llm <- chat_openai(model = "gpt-4o-mini", system_prompt = "You are a 72-year-old white man, politically conservative, living in rural Alabama, working as a dairy farmer.") my_llm$chat(top_issue) my_llm <- chat_openai(model = "gpt-4o-mini", system_prompt = "You are a 35-year-old Mexican immigrant, political Independent, living in New Mexico, working in construction.") my_llm$chat(top_issue) # How extreme are the positions that the LLM is willing to take? # Note that if we push too hard, the model may refuse answer or hedge/soften. my_llm <- chat_openai(model = "gpt-4o-mini", system_prompt = "You are a 40-year-old white nationalist living in rural Montana.") my_llm$chat("What are your views on immigration? Answer in 2 sentences.") my_llm <- chat_openai(model = "gpt-4o-mini", system_prompt = "You are a 40-year-old undocumented immigrant from Guatemala.") my_llm$chat("What are your views on immigration? Answer in 2 sentences.") # ================ Back to our Guardian data ================ # NOTE: If you do NOT want to get a Guardian API key, skip this entire section. # At the start of the next section, you'll load the pre-saved data from Posit Cloud. # Store the key in your console (not in a saved script): # Sys.setenv(GU_API_KEY = "your-gu-key") # Verify your key is loaded: nzchar(Sys.getenv("GU_API_KEY")) # gu_content() retrieves recent Guardian articles. # 'from_date' gives us starting data (today-3 days) # 'show_fields' specifies which data fields to include (all) # 'tag' specifies to only download news tagged as 'us-news' guardian <- gu_content(tag = "us-news/us-news", from_date = Sys.Date() - 7, show_fields = "all") # What sections did we get, and how many articles each? count(guardian, section_name) # We will keep 3 articles per section from our Guardian data: guardian <- group_by(guardian, section_name) # Group by section guardian <- filter(guardian, n() >= 3) # Drop sections with <3 articles guardian <- slice_head(guardian, n = 3) # Keep the first 3 articles per section guardian <- arrange(guardian, section_name) # Order by section guardian <- ungroup(guardian) count(guardian, section_name) dim(guardian) # Save the data for future use: saveRDS(guardian, "guardian_news_data.rds") # ================ Prompt Engineering ================ # Let's go back to our Guardian news story data: guardian <- readRDS("guardian_news_data.rds") # Remind yourself what it looks like: View(guardian) # Last week we used a single annotation prompt. # Today we'll see how much prompt design affects our results. # We compare three versions of the classification prompt: # (1) Minimal -- just the task # (2) Detailed -- role, constraints, detailed instructions # (3) Few-shot -- detailed + examples of correct classifications # Let's prepare our test dataset and our example articles in advance: world_news <- grep("World news", guardian$section_name) us_news <- grep("US news", guardian$section_name) business <- grep("Business", guardian$section_name) examples <- c(world_news[1], us_news[1], business[1]) examples example_data <- guardian[ examples, c("web_title", "section_name", "body_text")] test_data <- guardian[-examples, c("web_title", "section_name", "body_text")] example_data section_options <- paste(unique(test_data$section_name), collapse = ", ") section_options # Now, let's specify the object we want returned from the LLM: ann_type <- type_object("News article annotation", section = type_string("The news section"), confidence = type_integer("Confidence in classification, 1-100"), summary = type_string("A one-sentence summary of the article"), entities = type_array(description = "Named entities mentioned in the article", items = type_string())) # ------~~ Prompt 1: Minimal -------- # Minimum instructions; no role, no constraints, no examples. prompt_minimal <- paste("What news section does this article belong to? Choose one of:", section_options, "Also provide a confidence score (1-100), a one-sentence summary, and a list of named entities.") test_data$prompt_v1 <- paste(prompt_minimal, "TITLE:", test_data$web_title, "ARTICLE:", test_data$body_text) my_llm <- chat_openai(model = "gpt-4o-mini", params = params(temperature = 0)) result_v1 <- parallel_chat_structured(my_llm, type = ann_type, prompts = as.list(test_data$prompt_v1) ) test_data$llm_section_v1 <- result_v1$section test_data$llm_confidence_v1 <- result_v1$confidence accuracy_v1 <- mean(test_data$llm_section_v1 == test_data$section_name, na.rm = TRUE) confidence_v1 <- mean(test_data$llm_confidence_v1, na.rm = TRUE) accuracy_v1 confidence_v1 # ------~~ Prompt 2: Detailed -------- # Add a role, explicit constraints, and clearer instructions. prompt_detailed <- paste("Your task is to classify a Guardian news article into exactly one of these sections:", section_options, ". Choose the section that best reflects the article's primary topic according to an editor -- not a secondary theme. Assess your confidence from 1 (very uncertain) to 100 (certain). Summarize the article in one sentence of no more than 30 words. Extract all named entities (people, organizations, locations). Do not include dates or years in the entity list.") test_data$prompt_v2 <- paste(prompt_detailed, "TITLE:", test_data$web_title, "ARTICLE:", test_data$body_text) my_llm <- chat_openai(model = "gpt-4o-mini", params = params(temperature = 0), system_prompt = "You are a news classification assistant for The Guardian") result_v2 <- parallel_chat_structured(my_llm, type = ann_type, prompts = as.list(test_data$prompt_v2) ) test_data$llm_section_v2 <- result_v2$section test_data$llm_confidence_v2 <- result_v2$confidence accuracy_v2 <- mean(test_data$llm_section_v2 == test_data$section_name, na.rm = TRUE) confidence_v2 <- mean(test_data$llm_confidence_v2, na.rm = TRUE) accuracy_v2 confidence_v2 # ------~~ Prompt 3: Few-shot -------- # Add examples of correct classifications before the article to classify. # Few-shot examples help the model understand the task's nuances -- # especially for categories that are hard to define in words. example_1 <- paste("EXAMPLE 1:", "TITLE EXAMPLE 1:", example_data$web_title[1], "ARTICLE EXAMPLE 1:", example_data$body_text[1], "CORRECT SECTION EXAMPLE 1:", example_data$section_name[1]) example_2 <- paste("EXAMPLE 2:", "TITLE EXAMPLE 2:", example_data$web_title[2], "ARTICLE EXAMPLE 2:", example_data$body_text[2], "CORRECT SECTION EXAMPLE 2:", example_data$section_name[2]) example_3 <- paste("EXAMPLE 3:", "TITLE EXAMPLE 3:", example_data$web_title[3], "ARTICLE EXAMPLE 3:", example_data$body_text[3], "CORRECT SECTION EXAMPLE 3:", example_data$section_name[3]) prompt_fewshot <- paste("Your task is to classify a Guardian news article into exactly one of these sections:", section_options, ". Here are three examples of correct classifications:", example_1, example_2, example_3, " END OF EXAMPLE ARTICLES. Now classify the article below using the same logic. Choose the section that best reflects the article's primary topic according to an editor -- not a secondary theme. Assess your confidence from 1 (very uncertain) to 100 (certain). Summarize the article in one sentence of no more than 30 words. Extract all named entities (people, organizations, locations). Do not include dates or years in the entity list.") test_data$prompt_v3 <- paste(prompt_fewshot, "TITLE:", test_data$web_title, "ARTICLE:", test_data$body_text) my_llm <- chat_openai(model = "gpt-4o-mini", params = params(temperature = 0), system_prompt = "You are a news classification assistant for The Guardian") result_v3 <- parallel_chat_structured(my_llm, type = ann_type, prompts = as.list(test_data$prompt_v3) ) test_data$llm_section_v3 <- result_v3$section test_data$llm_confidence_v3 <- result_v3$confidence accuracy_v3 <- mean(test_data$llm_section_v3 == test_data$section_name, na.rm = TRUE) confidence_v3 <- mean(test_data$llm_confidence_v3, na.rm = TRUE) accuracy_v3 confidence_v3 # ------~~ Compare prompt versions -------- prompt_comparison <- data.frame( prompt = c("Minimal", "Structured", "Few-shot"), accuracy = round(c(accuracy_v1, accuracy_v2, accuracy_v3), 3) ) prompt_comparison # Prompt design matters -- but often less than model choice. # The Guardian's overlapping sections create a ceiling that better prompts # can nudge but not break through. # ================ Model Comparison ================ # Does it matter which model we use? # We run the detailed prompt on three models and compare -- # not just for accuracy, but for where they disagree. # ------~~ gpt-4o-mini -------- my_llm <- chat_openai(model = "gpt-4o-mini", params = params(temperature = 0), system_prompt = "You are a news classification assistant for The Guardian") result_gpt40 <- parallel_chat_structured(my_llm, type = ann_type, prompts = as.list(test_data$prompt_v2) ) test_data$llm_section_gpt40 <- result_gpt40$section test_data$llm_confidence_gpt40 <- result_gpt40$confidence accuracy_gpt40 <- mean(test_data$llm_section_gpt40 == test_data$section_name, na.rm = TRUE) confidence_gpt40 <- mean(test_data$llm_confidence_gpt40, na.rm = TRUE) accuracy_gpt40 confidence_gpt40 # ------~~ gpt-4.1 -------- my_llm <- chat_openai(model = "gpt-4.1", params = params(temperature = 0), system_prompt = "You are a news classification assistant for The Guardian") result_gpt41 <- parallel_chat_structured(my_llm, type = ann_type, prompts = as.list(test_data$prompt_v2) ) test_data$llm_section_gpt41 <- result_gpt41$section test_data$llm_confidence_gpt41 <- result_gpt41$confidence accuracy_gpt41 <- mean(test_data$llm_section_gpt41 == test_data$section_name, na.rm = TRUE) confidence_gpt41 <- mean(test_data$llm_confidence_gpt41, na.rm = TRUE) accuracy_gpt41 confidence_gpt41 # ------~~ Gemini 2.5 Flash Lite -------- my_llm <- chat_google_gemini(model = "gemini-2.5-flash-lite") result_gemini <- parallel_chat_structured(my_llm, type = ann_type, prompts = as.list(test_data$prompt_v2) ) test_data$llm_section_gemini <- result_gemini$section test_data$llm_confidence_gemini <- result_gemini$confidence accuracy_gemini <- mean(test_data$llm_section_gemini == test_data$section_name, na.rm = TRUE) confidence_gemini <- mean(test_data$llm_confidence_gemini, na.rm = TRUE) accuracy_gemini confidence_gemini # ------~~ Compare models -------- model_comparison <- data.frame( model = c("gpt-4o-mini", "gpt-4.1", "gemini-2.5-flash-lite"), accuracy = c(accuracy_gpt40, accuracy_gpt41, accuracy_gemini), confidence = c(confidence_gpt40, confidence_gpt41, confidence_gemini) ) model_comparison # How often do the models agree with each other? mean(test_data$llm_section_gpt40 == test_data$llm_section_gpt41, na.rm = TRUE) mean(test_data$llm_section_gpt40 == test_data$llm_section_gemini, na.rm = TRUE) mean(test_data$llm_section_gpt41 == test_data$llm_section_gemini, na.rm = TRUE) # Are there any cases where all three models disagree with each other? all_disagree <- (test_data$llm_section_gpt40 != test_data$llm_section_gpt41) & (test_data$llm_section_gpt40 != test_data$llm_section_gemini) & (test_data$llm_section_gpt41 != test_data$llm_section_gemini) # These would be the hardest cases in our dataset: test_data[all_disagree, c("web_title", "section_name")] # Are there any cases where all three models agree with each other? all_agree <- (test_data$llm_section_gpt40 == test_data$llm_section_gpt41) & (test_data$llm_section_gpt40 == test_data$llm_section_gemini) & (test_data$llm_section_gpt41 == test_data$llm_section_gemini) # These would be the most straightforward cases in our dataset: test_data[all_agree, c("web_title", "section_name")] # ================ Validation ================ # How do we know if our classification is working well? # Let's use package 'caret' and look at a few metrics! # We use the gpt-4.1 annotations as our main result: test_data$llm_section <- test_data$llm_section_gpt41 test_data$llm_confidence <- test_data$llm_confidence_gpt41 # ------~~ Confusion matrix -------- # The confusionMatrix() function from 'caret' calculates all the metrics we need. # It does insist that we feed it factors (categorical variables) though. factor_levels <- unique(test_data$section_name) cm <- confusionMatrix(factor(test_data$llm_section, levels = factor_levels ), factor(test_data$section_name, levels = factor_levels) ) # Rows = predicted, columns = true. Off-diagonal entries = errors. cm$table # ------~~ Accuracy and kappa -------- # Accuracy gives us the proportion of articles we correctly classified. # However, just looking at accuracy doesn't account for chance agreement. # Cohen's kappa tells us how much scores agree over what's expected by chance. # # Interpretation (Landis & Koch 1977): # < 0.20: slight # 0.21-0.40: fair # 0.41-0.60: moderate # 0.61-0.80: substantial # > 0.80: almost perfect cm$overall["Accuracy"] cm$overall["Kappa"] # ------~~ Precision, recall, and F1 -------- # Precision: of all articles the model labeled as X, how many truly were X? # Low precision = model over-applies the label (too many false positives) # # Recall: of all articles that truly belong to X, how many did the model find? # Low recall = model misses true cases (too many false negatives) # # F1: harmonic mean of precision and recall -- balances both into one number. # A section with F1 = 1.0 is perfect; F1 = 0 means complete failure. # If you are looking for all examples of a rare category (hate speech, misinformation), # recall matters most -- you don't want to miss any cases. # If you're building a clean labeled dataset, precision matters most # every labeled case should be a true case. cm$byClass[, c("Precision", "Recall", "F1")] # ------~~ Model confidence -------- # If confidence were well-calibrated, correct predictions would have # systematically higher confidence than incorrect ones. # In practice, LLMs tend to be overconfident regardless of accuracy. llm_accurate <- (test_data$llm_section == test_data$section_name) mean(test_data$llm_confidence[llm_accurate]) mean(test_data$llm_confidence[!llm_accurate]) # ================ Web Search ================ # ellmer provides built-in web search tools for major providers. # When registered, the model decides when to search and incorporates # the results into its response. # Note that web search would increase the cost of your requests. # ------~~ Basic web search -------- my_llm <- chat_openai(model = "gpt-4o-mini") my_llm$register_tool(openai_tool_web_search()) my_llm$chat("What are the top 3 US news stories today? Describe each in one sentence") # For Gemini: my_llm <- chat_google_gemini(model = "gemini-2.5-flash-lite") my_llm$register_tool(google_tool_web_search()) my_llm$chat("What are the top 3 US news stories today? Describe each in one sentence") # Note that without search, you may not get good answers on recent events. recent_event <- "Are there planned changes in the leadership of Apple?" my_llm_nosearch <- chat_openai(model = "gpt-4o-mini") my_llm_nosearch$chat(recent_event) my_llm_search <- chat_openai(model = "gpt-4o-mini") my_llm_search$register_tool(openai_tool_web_search()) my_llm_search$chat(recent_event) # ================ Image Annotation ================ # We ran short on time last week -- so let's do images now! # The Guardian returns a thumbnail image for each article in the 'thumbnail' column. # We can use these images for annotation -- and since we know each article's # section, we have ground truth context for the images too. # Let's look at a random Guardian article and its image: guardian$web_title[10] guardian$section_name[10] guardian$thumbnail[10] # Open the link to the thumbnail so we can see it: browseURL(guardian$thumbnail[10]) my_llm <- chat_openai(model = "gpt-4o-mini") my_llm$chat(content_image_url(guardian$thumbnail[10]), "Describe this image in 1 sentence") # ------~~ Can you see a human? -------- # Starting simple: a binary question about image content. human_prompt <- "Is there a human clearly visible in this image? Answer only Yes or No." my_llm <- chat_openai(model = "gpt-4o-mini") my_llm$chat(content_image_url(guardian$thumbnail[10]), human_prompt) # Ask the same question across several images: # (note that we could also do this with parallel_chat) for (i in 1:5) { my_llm <- chat_openai(model = "gpt-4o-mini") cat( "\n\n", guardian$web_title[i], " ------ ") my_llm$chat(content_image_url(guardian$thumbnail[i]), human_prompt) } # ------~~ Identify the public figure -------- # Let's try identifying a few well-known public figures: image_person_prompt <- "Look at this image carefully. If a well-known public figure or government official is visible, return their full name. If you cannot recognize the person, say only: Unidentified. If no person is visible, say only: No person visible." my_llm <- chat_openai(model = "gpt-4o-mini") my_llm$chat(content_image_file("DT.jpg"), image_person_prompt) my_llm <- chat_openai(model = "gpt-4o-mini") my_llm$chat(content_image_file("JB.jpg"), image_person_prompt) my_llm <- chat_openai(model = "gpt-4o-mini") my_llm$chat(content_image_file("BO.jpg"), image_person_prompt) # As you can see, Open AI's policy generally bans identifying people. # Let's see if Gemini will be more responsive to our request. # Gemini does not allow image URLs, only local image files. my_llm <- chat_google_gemini(model = "gemini-2.5-flash-lite") my_llm$chat(content_image_file("DT.jpg"), image_person_prompt) my_llm <- chat_google_gemini(model = "gemini-2.5-flash-lite") my_llm$chat(content_image_file("JB.jpg"), image_person_prompt) my_llm <- chat_google_gemini(model = "gemini-2.5-flash-lite") my_llm$chat(content_image_file("BO.jpg"), image_person_prompt) # ------~~ Guess the emotion! -------- image_em_prompt <- "If a person is visible in this image, describe the primary emotion expressed on their face. Reply with only a single word. For example: sad, neutral, serious, happy, angry, concerned, or surprised. If no person is visible, say: NA" my_llm <- chat_openai(model = "gpt-4o-mini") my_llm$chat(content_image_file("DT.jpg"), image_em_prompt) my_llm <- chat_openai(model = "gpt-4o-mini") my_llm$chat(content_image_file("JB.jpg"), image_em_prompt) my_llm <- chat_openai(model = "gpt-4o-mini") my_llm$chat(content_image_file("BO.jpg"), image_em_prompt) # ------~~ What breed is this puppy? -------- # Now let's do something important: look at puppy pictures! dog_url <- "https://upload.wikimedia.org/wikipedia/commons/6/68/Szczenie_Jack_Russell_Terrier3.jpg" my_llm <- chat_openai(model = "gpt-4o-mini") my_llm$chat(content_image_url(dog_url), "What breed or breeds of dog are visible in this image? Name each breed you can identify. Respond with only breed names.") # ------~~ Identify meme politics -------- # Now we ask the model to do something genuinely hard: decode political irony. # Meme classification requires cultural knowledge, sarcasm detection, and # an understanding of the political environment. Complicated for machine coding! meme_type <- type_object("Political meme evaluation", view = type_string("Political leaning" ), reason = type_string("Two-sentence politics reasoning" ), confidence = type_integer("Confidence in the political leaning, 1-100" ), rating = type_integer("Persuasiveness score from 1-10" ), title = type_string("An appropriate title for the meme" ) ) meme_prompt <- "What political viewpoint does this meme express? Reply with one word: liberal, conservative, or neither. Evaluate your confidence in the political leaning, 1 to 100. Then, explain your reasoning in 2 sentences. Score the meme's persuasiveness from 1 (least) to 10 (most). Finally, give the meme an appropriate short title." my_llm <- chat_openai(model = "gpt-4o-mini") my_llm$chat_structured(content_image_file("meme_1.jpg"), meme_prompt, type = meme_type) my_llm <- chat_openai(model = "gpt-4o-mini") my_llm$chat_structured(content_image_file("meme_2.jpg"), meme_prompt, type = meme_type) my_llm <- chat_openai(model = "gpt-4o-mini") my_llm$chat_structured(content_image_file("meme_3.jpg"), meme_prompt, type = meme_type) # ================ THE END ================